import pyarrow.parquet as pq
import numpy as np

def load_in_file(filename):
    parquet_file = pq.ParquetFile(filename)
    data = parquet_file.read().to_pandas()
    return data

def delunstatiscolumns(normal,abnormal,threshold = 0.1):
    todel = []
    for i in normal.columns:
        if i in abnormal.columns:
            diff = normal[i].values-abnormal[i].values
            maxval = np.max([np.abs(normal[i].values),np.abs(abnormal[i].values)],axis=0)
            if np.max(np.abs(diff)/(maxval+0.001)) < threshold:
                todel.append(i)
    return todel
